package priv.lirenhe.dissertation.executor;

import java.io.File;
import java.io.FileOutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.io.FileUtils;

import priv.lirenhe.dissertation.bdb.util.UrlQueUtils;
import priv.lirenhe.dissertation.cacher.Cacher;
import priv.lirenhe.dissertation.cacher.CacherManager;
import priv.lirenhe.dissertation.component.crawler_robot.url_manager.url_deduplication.UrlQueueDeduplication;
import priv.lirenhe.dissertation.component.data_parser.data_parser_executor.DataParserExecutor;
import priv.lirenhe.dissertation.component.data_parser.util.UrlUtil;
import priv.lirenhe.dissertation.component.search_engine.search_engine_executor.SearchEngineExecutor;

public class GetAndCleanUpData {
	
	public void putDataInMysqlDb(){
		
		CacherManager cacherManager = new CacherManager();
		Map<String, Cacher> dataCacherOfMap = null;
		Cacher webContentCacher = null;
		String realUrl = null;
		String webContent = null;
		
		Map<Object,Object> dataParserExecutorMap = null;
		List<String> urlsOfList = null;
		String webContentInText = null;
		String webTitle = null;
		
		Map<String,Map<String,String>> webUrlMap = new HashMap<String,Map<String,String>>();
		Map<String,String> webPageMap = new HashMap<String,String>();
		Map<String,String> paramsOfWebUrlMap = null;
		
		int cacheFlushNum = 0;
		
		while(true){
			System.out.println(Thread.currentThread().getName()+ ":扫描缓存" + ++cacheFlushNum);
			try{
				if(!cacherManager.isEmpty()){
					Map<String,String> map_index = null;
					
					System.out.println(Thread.currentThread().getName()+ ":缓存不为空");
					
					UrlQueueDeduplication urlQueueDeduplication = new UrlQueueDeduplication();
					
					dataCacherOfMap = cacherManager.getCacheAll();
					System.out.println("获取缓存的数量："+dataCacherOfMap.size());
					
					List<Map<String,String>> list_map_OfWebPage = new ArrayList<Map<String,String>>();//建立索引的数据集合
					DataParserExecutor dataParserExecutor = null;
					for(Map.Entry<String, Cacher> entry:dataCacherOfMap.entrySet()){
						
						map_index = new HashMap<String,String>();
						
						realUrl = entry.getKey();
						webContentCacher = entry.getValue();
						webContent = (String)webContentCacher.getDatas();
						
						if(realUrl != null && webContent != null){
							dataParserExecutor = new DataParserExecutor(realUrl, webContent);
							dataParserExecutorMap = dataParserExecutor.getDataAfterParser();
							if(dataParserExecutor != null){
								urlsOfList = (List<String>) dataParserExecutorMap.get("urlOfList");
								webContentInText = (String) dataParserExecutorMap.get("webContentInText");
								webTitle = (String) dataParserExecutorMap.get("webTitle");
								
//								BufferedWriter bw = new BufferedWriter(OutputStreamWriter)
//								FileOutputStream fos = FileUtils.openOutputStream(new File("C:\\Users\\lenovo\\Desktop\\html-jsp\\htmls\\html01.txt"), true);
								String space = "\r\n";
								FileUtils.write(new File("C:\\Users\\lenovo\\Desktop\\html-jsp\\htmls\\html01.txt"), webContentInText+space, true);
							}
						}
						if(urlsOfList != null && !urlsOfList.isEmpty()){
							for(String str:urlsOfList){
								if(!urlQueueDeduplication.usedUrlQueHasUsedThisUrl(str) && !urlQueueDeduplication.notUsedUrlQueHasUsedThisUrl(str)){
									
									//url进入BDB
									if(str != null)
										UrlInBDB.putUrlInBDB(str);
									else
										continue;
									if(str.length()>255){
										System.out.println("url长度大于255"+str);
									}
									
									paramsOfWebUrlMap = new HashMap<String,String>();
									paramsOfWebUrlMap.put("webUrlRealUrl",str);
									paramsOfWebUrlMap.put("webUrlMainfatherurl", UrlUtil.resolveMainUrl(str));
									paramsOfWebUrlMap.put("webUrlFatherurl",realUrl);
									paramsOfWebUrlMap.put("webUrlEffective", "effective");
									paramsOfWebUrlMap.put("webUrlHasused","notHasUsed");
									webUrlMap.put(str, paramsOfWebUrlMap);
								}else{
									System.out.println("在BDB中重复url="+str);
									continue;
								}
							}
							//刷新BDB数据库
							if(UrlQueUtils.getNotUsedUrlQueue() != null){
								UrlQueUtils.syncNotUsedUrlQueueBDB();//持久化到磁盘，等到以后分开写的时候可以在finally中持久化
							}
						}
						
						//webpage数据的解析
						webPageMap.put("webPageRealurl",realUrl);
						if(webContentInText != null && webTitle == null){
							if(webContentInText.length() > 15){
								webTitle = webContentInText.substring(0,15);
							}
						}
						webPageMap.put("webPageTitle", webTitle);
						webPageMap.put("webPageContentCode",webContent);
						webPageMap.put("webPageContentText", webContentInText);
						
						if(webContentInText != null){
							map_index.putAll(webPageMap);
							list_map_OfWebPage.add(map_index);
						}
						
						//入库
						System.out.println("入库===========================");
						InMysqlDB.putDataInMysqlDB(webUrlMap, webPageMap);
						System.out.println("入库出来===============");
					}
					
					//建立索引
					System.out.println("建立索引====");
					SearchEngineExecutor searchEngineExecutor = new SearchEngineExecutor();
					searchEngineExecutor.searchEngineExecute(list_map_OfWebPage);
					
				}else{
					try {
						System.out.println(Thread.currentThread().getName() + ":缓存为空,睡眠五秒");
						
						Thread.sleep(5000);
					} catch (InterruptedException e) {
						e.printStackTrace();
					}
				}
				
			}catch(Exception e){
				e.printStackTrace();
			}
		}
	}
}
